{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1 数据读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 1.1 安装pyarrow包，用于读取数据\n",
    "!pip install pandas pyarrow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 2.2 使用pyarrow包读取数据\n",
    "import pandas as pd\n",
    "\n",
    "df_train = pd.read_parquet('train.parquet')\n",
    "df_test = pd.read_parquet('test.parquet')\n",
    "# 现在 df 是一个 Pandas DataFrame，你可以像操作其他 DataFrame 一样操作它"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prompt</th>\n",
       "      <th>response_a</th>\n",
       "      <th>response_b</th>\n",
       "      <th>winner</th>\n",
       "      <th>model_a</th>\n",
       "      <th>model_b</th>\n",
       "      <th>language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>00007cff95d7f7974642a785aca248b0f26e60d3312fac...</td>\n",
       "      <td>vieš po Slovensky?</td>\n",
       "      <td>Áno, hovorím po slovensky. Ako vám môžem pomôcť?</td>\n",
       "      <td>Áno, veď som tu! Môžem ti pomôcť s otázkami al...</td>\n",
       "      <td>model_a</td>\n",
       "      <td>o1-preview</td>\n",
       "      <td>reka-core-20240904</td>\n",
       "      <td>Slovak</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>00010ed04b536f56ebe43eef1100c13906abea12bf9855...</td>\n",
       "      <td>You will be given a piece of news. Analyze it ...</td>\n",
       "      <td>Let's break down the news and analyze it accor...</td>\n",
       "      <td>```json\\n{\\n  \"contains_orgs\": true,\\n  \"orgs\"...</td>\n",
       "      <td>model_a</td>\n",
       "      <td>gemma-2-27b-it</td>\n",
       "      <td>gemini-1.5-flash-002</td>\n",
       "      <td>Russian</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0003800d510e38803efba5ceaec122bc66408fe367b0be...</td>\n",
       "      <td>Dört basamaklı, rakamları birbirinden ve sıfır...</td>\n",
       "      <td>Bu soruyu çözmek için, verilen koşulları adım ...</td>\n",
       "      <td>Bu problemi adım adım çözelim:\\n\\n1) ABCD - DC...</td>\n",
       "      <td>model_a</td>\n",
       "      <td>gpt-4-0125-preview</td>\n",
       "      <td>claude-3-5-sonnet-20240620</td>\n",
       "      <td>Turkish</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>00072026c68f5418ef2da238394e418ce72a534b9b22d5...</td>\n",
       "      <td>현재 추천된 탑 3 종목인 Cabaletta Bio (CABA), Rocket Ph...</td>\n",
       "      <td>죄송하지만 저는 금융 조언을 제공할 수 없습니다. 저는 AI 모델이며, 투자 결정에...</td>\n",
       "      <td>현재 추천된 탑 3 종목에 순위를 매기기 위해서는 여러 가지 요소들을 고려해야 합니...</td>\n",
       "      <td>model_b</td>\n",
       "      <td>gemma-2-2b-it</td>\n",
       "      <td>llama-3.1-nemotron-70b-instruct</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0007ce7cf6bc1b5a8f8a4669b854fb12030863c970d9dc...</td>\n",
       "      <td>Please be boring</td>\n",
       "      <td>Alright, I'll be as boring as possible.\\n\\nTod...</td>\n",
       "      <td>Understood. Here is a straightforward, unadorn...</td>\n",
       "      <td>model_a</td>\n",
       "      <td>reka-flash-20240722</td>\n",
       "      <td>grok-2-2024-08-13</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48434</th>\n",
       "      <td>fff5962462043091a771018e37ece11c425e9c7cca64b2...</td>\n",
       "      <td>你好，请你介绍下你自己</td>\n",
       "      <td>你好！我是LSK，一个形式为人工智能的chatbot。</td>\n",
       "      <td>### 介绍\\n\\n我是一款人工智能语言模型，基于深度学习技术开发而成。我的主要功能是理解自...</td>\n",
       "      <td>model_b</td>\n",
       "      <td>llama-3.2-3b-instruct</td>\n",
       "      <td>llama-3.1-8b-instruct</td>\n",
       "      <td>Chinese</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48435</th>\n",
       "      <td>fffad6c95fb82e3a694b4952965852130285e0a38905d5...</td>\n",
       "      <td>Viva Technology参展费用</td>\n",
       "      <td>Viva Technology是一个国际性的科技展览会，每年在法国巴黎举行。参展费用会根据多...</td>\n",
       "      <td>我无法找到Viva Technology参展费用的具体公开信息。\\n\\n**参展费用通常取决...</td>\n",
       "      <td>model_a</td>\n",
       "      <td>reka-core-20240904</td>\n",
       "      <td>gemini-1.5-flash-exp-0827</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48436</th>\n",
       "      <td>fffbf502f2cd88b706a4bd6e54ed90e9a2d59d1643140d...</td>\n",
       "      <td>다음은 '사무장병원 경찰 조사 동행기 - 장효강 변호사의 현장 이야기'라는 제목의 ...</td>\n",
       "      <td>사무장병원 경찰 조사 동행기 - 장효강 변호사의 현장 이야기\\n\\n**\"압수수색, ...</td>\n",
       "      <td>도입부를 개선하기 위해서는 독자의 관심을 더 끌어야 하며, 좀 더 명료하고 간결하게...</td>\n",
       "      <td>model_a</td>\n",
       "      <td>gemini-1.5-pro-002</td>\n",
       "      <td>yi-lightning-lite</td>\n",
       "      <td>Korean</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48437</th>\n",
       "      <td>fffd2ffdaa03e9e0a0cd1e8f2ee80f530bb19b08fa4312...</td>\n",
       "      <td>What is the height of the church located in Sa...</td>\n",
       "      <td>**Church Information: Église de Saint-Nazaire-...</td>\n",
       "      <td>I couldn't find any information about a church...</td>\n",
       "      <td>model_a</td>\n",
       "      <td>llama-3.1-70b-instruct</td>\n",
       "      <td>llama-3.2-3b-instruct</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>48438</th>\n",
       "      <td>ffff059aea247f1dc7a09cfea55e00309b5b9a2e8cd9fc...</td>\n",
       "      <td>1000Tb 每秒传输10gb要多久才能完成</td>\n",
       "      <td>**1. 单位转换:**\\n\\n* 1000 Tb = 1000 * 1024 Gb = 1...</td>\n",
       "      <td>要计算传输 1000TB 数据以 10Gb/s 的速率需要多长时间，我们可以按以下步骤进行。...</td>\n",
       "      <td>model_a</td>\n",
       "      <td>gemini-1.5-flash-exp-0827</td>\n",
       "      <td>chatgpt-4o-latest-20240903</td>\n",
       "      <td>unknown</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>48439 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                      id  \\\n",
       "0      00007cff95d7f7974642a785aca248b0f26e60d3312fac...   \n",
       "1      00010ed04b536f56ebe43eef1100c13906abea12bf9855...   \n",
       "2      0003800d510e38803efba5ceaec122bc66408fe367b0be...   \n",
       "3      00072026c68f5418ef2da238394e418ce72a534b9b22d5...   \n",
       "4      0007ce7cf6bc1b5a8f8a4669b854fb12030863c970d9dc...   \n",
       "...                                                  ...   \n",
       "48434  fff5962462043091a771018e37ece11c425e9c7cca64b2...   \n",
       "48435  fffad6c95fb82e3a694b4952965852130285e0a38905d5...   \n",
       "48436  fffbf502f2cd88b706a4bd6e54ed90e9a2d59d1643140d...   \n",
       "48437  fffd2ffdaa03e9e0a0cd1e8f2ee80f530bb19b08fa4312...   \n",
       "48438  ffff059aea247f1dc7a09cfea55e00309b5b9a2e8cd9fc...   \n",
       "\n",
       "                                                  prompt  \\\n",
       "0                                     vieš po Slovensky?   \n",
       "1      You will be given a piece of news. Analyze it ...   \n",
       "2      Dört basamaklı, rakamları birbirinden ve sıfır...   \n",
       "3      현재 추천된 탑 3 종목인 Cabaletta Bio (CABA), Rocket Ph...   \n",
       "4                                      Please be boring    \n",
       "...                                                  ...   \n",
       "48434                                        你好，请你介绍下你自己   \n",
       "48435                                Viva Technology参展费用   \n",
       "48436  다음은 '사무장병원 경찰 조사 동행기 - 장효강 변호사의 현장 이야기'라는 제목의 ...   \n",
       "48437  What is the height of the church located in Sa...   \n",
       "48438                             1000Tb 每秒传输10gb要多久才能完成   \n",
       "\n",
       "                                              response_a  \\\n",
       "0       Áno, hovorím po slovensky. Ako vám môžem pomôcť?   \n",
       "1      Let's break down the news and analyze it accor...   \n",
       "2      Bu soruyu çözmek için, verilen koşulları adım ...   \n",
       "3      죄송하지만 저는 금융 조언을 제공할 수 없습니다. 저는 AI 모델이며, 투자 결정에...   \n",
       "4      Alright, I'll be as boring as possible.\\n\\nTod...   \n",
       "...                                                  ...   \n",
       "48434                        你好！我是LSK，一个形式为人工智能的chatbot。   \n",
       "48435  Viva Technology是一个国际性的科技展览会，每年在法国巴黎举行。参展费用会根据多...   \n",
       "48436  사무장병원 경찰 조사 동행기 - 장효강 변호사의 현장 이야기\\n\\n**\"압수수색, ...   \n",
       "48437  **Church Information: Église de Saint-Nazaire-...   \n",
       "48438  **1. 单位转换:**\\n\\n* 1000 Tb = 1000 * 1024 Gb = 1...   \n",
       "\n",
       "                                              response_b   winner  \\\n",
       "0      Áno, veď som tu! Môžem ti pomôcť s otázkami al...  model_a   \n",
       "1      ```json\\n{\\n  \"contains_orgs\": true,\\n  \"orgs\"...  model_a   \n",
       "2      Bu problemi adım adım çözelim:\\n\\n1) ABCD - DC...  model_a   \n",
       "3      현재 추천된 탑 3 종목에 순위를 매기기 위해서는 여러 가지 요소들을 고려해야 합니...  model_b   \n",
       "4      Understood. Here is a straightforward, unadorn...  model_a   \n",
       "...                                                  ...      ...   \n",
       "48434  ### 介绍\\n\\n我是一款人工智能语言模型，基于深度学习技术开发而成。我的主要功能是理解自...  model_b   \n",
       "48435  我无法找到Viva Technology参展费用的具体公开信息。\\n\\n**参展费用通常取决...  model_a   \n",
       "48436  도입부를 개선하기 위해서는 독자의 관심을 더 끌어야 하며, 좀 더 명료하고 간결하게...  model_a   \n",
       "48437  I couldn't find any information about a church...  model_a   \n",
       "48438  要计算传输 1000TB 数据以 10Gb/s 的速率需要多长时间，我们可以按以下步骤进行。...  model_a   \n",
       "\n",
       "                         model_a                          model_b language  \n",
       "0                     o1-preview               reka-core-20240904   Slovak  \n",
       "1                 gemma-2-27b-it             gemini-1.5-flash-002  Russian  \n",
       "2             gpt-4-0125-preview       claude-3-5-sonnet-20240620  Turkish  \n",
       "3                  gemma-2-2b-it  llama-3.1-nemotron-70b-instruct  English  \n",
       "4            reka-flash-20240722                grok-2-2024-08-13  English  \n",
       "...                          ...                              ...      ...  \n",
       "48434      llama-3.2-3b-instruct            llama-3.1-8b-instruct  Chinese  \n",
       "48435         reka-core-20240904        gemini-1.5-flash-exp-0827  English  \n",
       "48436         gemini-1.5-pro-002                yi-lightning-lite   Korean  \n",
       "48437     llama-3.1-70b-instruct            llama-3.2-3b-instruct  English  \n",
       "48438  gemini-1.5-flash-exp-0827       chatgpt-4o-latest-20240903  unknown  \n",
       "\n",
       "[48439 rows x 8 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看训练集数据\n",
    "df_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>prompt</th>\n",
       "      <th>response_a</th>\n",
       "      <th>response_b</th>\n",
       "      <th>scored</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>327228</td>\n",
       "      <td>Caso Clínico: Un hombre de 70 años con anteced...</td>\n",
       "      <td>**Diagnóstico Diferencial de Anemia en Pacient...</td>\n",
       "      <td>Basándonos en el caso clínico presentado, pode...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1139415</td>\n",
       "      <td>Peel Company received a cash dividend from a ...</td>\n",
       "      <td>The correct answer is **(a) No   No**. Here's ...</td>\n",
       "      <td>The correct answer is **(a) No No**. Here's wh...</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1235630</td>\n",
       "      <td>Há um grave problema com o relógio da torre da...</td>\n",
       "      <td>Dois problemas interessantes!\\n\\n**Problema 1:...</td>\n",
       "      <td>Vamos resolver os dois problemas em sequência....</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                             prompt  \\\n",
       "0   327228  Caso Clínico: Un hombre de 70 años con anteced...   \n",
       "1  1139415   Peel Company received a cash dividend from a ...   \n",
       "2  1235630  Há um grave problema com o relógio da torre da...   \n",
       "\n",
       "                                          response_a  \\\n",
       "0  **Diagnóstico Diferencial de Anemia en Pacient...   \n",
       "1  The correct answer is **(a) No   No**. Here's ...   \n",
       "2  Dois problemas interessantes!\\n\\n**Problema 1:...   \n",
       "\n",
       "                                          response_b  scored  \n",
       "0  Basándonos en el caso clínico presentado, pode...   False  \n",
       "1  The correct answer is **(a) No No**. Here's wh...   False  \n",
       "2  Vamos resolver os dois problemas em sequência....   False  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 查看测试集数据\n",
    "df_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                                  id  \\\n",
      "0  00007cff95d7f7974642a785aca248b0f26e60d3312fac...   \n",
      "1  00010ed04b536f56ebe43eef1100c13906abea12bf9855...   \n",
      "2  0003800d510e38803efba5ceaec122bc66408fe367b0be...   \n",
      "3  00072026c68f5418ef2da238394e418ce72a534b9b22d5...   \n",
      "4  0007ce7cf6bc1b5a8f8a4669b854fb12030863c970d9dc...   \n",
      "\n",
      "                                              prompt  \\\n",
      "0                                 vieš po Slovensky?   \n",
      "1  You will be given a piece of news. Analyze it ...   \n",
      "2  Dört basamaklı, rakamları birbirinden ve sıfır...   \n",
      "3  현재 추천된 탑 3 종목인 Cabaletta Bio (CABA), Rocket Ph...   \n",
      "4                                  Please be boring    \n",
      "\n",
      "                                          response_a  \\\n",
      "0   Áno, hovorím po slovensky. Ako vám môžem pomôcť?   \n",
      "1  Let's break down the news and analyze it accor...   \n",
      "2  Bu soruyu çözmek için, verilen koşulları adım ...   \n",
      "3  죄송하지만 저는 금융 조언을 제공할 수 없습니다. 저는 AI 모델이며, 투자 결정에...   \n",
      "4  Alright, I'll be as boring as possible.\\n\\nTod...   \n",
      "\n",
      "                                          response_b   winner  \\\n",
      "0  Áno, veď som tu! Môžem ti pomôcť s otázkami al...  model_a   \n",
      "1  ```json\\n{\\n  \"contains_orgs\": true,\\n  \"orgs\"...  model_a   \n",
      "2  Bu problemi adım adım çözelim:\\n\\n1) ABCD - DC...  model_a   \n",
      "3  현재 추천된 탑 3 종목에 순위를 매기기 위해서는 여러 가지 요소들을 고려해야 합니...  model_b   \n",
      "4  Understood. Here is a straightforward, unadorn...  model_a   \n",
      "\n",
      "               model_a                          model_b language  \n",
      "0           o1-preview               reka-core-20240904   Slovak  \n",
      "1       gemma-2-27b-it             gemini-1.5-flash-002  Russian  \n",
      "2   gpt-4-0125-preview       claude-3-5-sonnet-20240620  Turkish  \n",
      "3        gemma-2-2b-it  llama-3.1-nemotron-70b-instruct  English  \n",
      "4  reka-flash-20240722                grok-2-2024-08-13  English  \n",
      "An error occurred: . I apply this linear rotation to option-implied variances (VIX2\n",
      ")\n",
      "rather than to volatilities because return variances combine linearly across matu\u0002rity (assuming no autocorrelation in returns) whereas volatilities do not 翻译 cannot be used in worksheets.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "try:\n",
    "    # 读取 Parquet 文件\n",
    "    df = pd.read_parquet('train.parquet')\n",
    "\n",
    "    # 显示 DataFrame 的前几行\n",
    "    print(df.head())\n",
    "\n",
    "    # 将 DataFrame 保存为 CSV 文件，设置转义字符为反斜杠\n",
    "    df.to_csv('train.csv', index=False, escapechar='\\\\')\n",
    "\n",
    "    # 将 DataFrame 保存为 Excel 文件\n",
    "    df.to_excel('train.xlsx', index=False)\n",
    "\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred: {e}\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "crawler",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
